# scrapy genspider -t crawl sun www.xxx.com
import scrapy
from scrapy.linkextractors import LinkExtractor
from scrapy.spiders import CrawlSpider, Rule
from sunPro.items import SunproItem,DetailItem

# 需求：爬取sun网站中的编号，新闻标题，新闻内容，标号
class SunSpider(CrawlSpider):
    name = "sun"
    # allowed_domains = ["www.xxx.com"]
    start_urls = ["https://www.xxx.com"]

    # 链接提取器：根据指定规则（allow="正则"）进行指定链接的提取
    link = LinkExtractor(allow=r"Items/")
    link_detail = LinkExtractor(allow=r"")

    rules = (
        # 规则解析器：将链接提取器提取到的链接进行指定规则（callback）的解析操作
        Rule(link, callback="parse_item", follow=True),
        # follow=True：可以将链接提取器 继续作用到 链接提取器提取到的链接 所对应的页面中
        Rule(link_detail, callback="parse_detail")
    )

    # 解析新闻编号和新闻的标题
    # 如下两个解析方法中是不可以实现请求传参！
    # 如下将两个解析方法解析的数据存储到同一个item中，可以依次存储到两个item
    def parse_item(self, response):
        # 注意：xpath表达式中不可以出现tbody标签
        tr_list = response.xpath("")
        for tr in tr_list:
            new_num = tr.xpath(".//div[@class='").extract_first()
            new_title = tr.xpath(".//div[@class='").extract_first()

            # print(new_num, new_title)
            item = SunproItem()
            item["title"] = new_title
            item["new_num"] = new_num

            yield item

    # 解析新闻内容和新闻编号
    def parse_detail(self, response):
        new_id = response.xpath("").extract_first()
        new_content = response.xpath("").extract()
        new_content = ''.join(new_content)

        # print(new_id, new_content)
        item = DetailItem()
        item["content"] = new_content
        item["new_id"] = new_id

        yield item